From 4ca62049b32fa960953811dfe31b85be742528ee Mon Sep 17 00:00:00 2001 From: tsteven4 <13596209+tsteven4@users.noreply.github.com> Date: Tue, 26 Jan 2021 06:42:40 -0700 Subject: [PATCH] add github actions for CI. (#662) * add github actions for CI. * fix shellcheck issues in ci_install_windows.sh * generalize windows ci build support cleanup powershell scripts with Invoke-ScriptAnalyzer, including use of "-Settings CodeFormatting". create staging directory for windows installer creation. It is difficult to exclude all the files in the build directory that we don't want to distribute as the files are a function of the build system. add windows cmake build. --- .github/workflows/fedora.yml | 32 +++++++ .github/workflows/macos.yml | 87 +++++++++++++++++ .github/workflows/ubuntu.yml | 96 +++++++++++++++++++ .github/workflows/windows.yml | 121 ++++++++++++++++++++++++ appveyor.yml | 5 +- gui/setup.iss | 15 ++- gui/setup.iss.in | 15 ++- gui/setup.iss.qmake.in | 15 ++- tools/Dockerfile | 4 +- tools/Dockerfile_f32 | 1 - tools/Dockerfile_f33 | 22 +++++ tools/Dockerfile_qtio | 4 +- tools/ci_install_windows.sh | 53 +++++++++++ tools/ci_setup_windows.ps1 | 55 +++++++++++ tools/make_windows_release.ps1 | 94 +++++++++++------- tools/uploadtool/upload_github.sh | 152 ++++++++++++++++++++++++++++++ 16 files changed, 705 insertions(+), 66 deletions(-) create mode 100644 .github/workflows/fedora.yml create mode 100644 .github/workflows/macos.yml create mode 100644 .github/workflows/ubuntu.yml create mode 100644 .github/workflows/windows.yml create mode 100644 tools/Dockerfile_f33 create mode 100755 tools/ci_install_windows.sh create mode 100755 tools/ci_setup_windows.ps1 create mode 100755 tools/uploadtool/upload_github.sh diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml new file mode 100644 index 000000000..22a339da0 --- /dev/null +++ b/.github/workflows/fedora.yml @@ -0,0 +1,32 @@ +name: "fedora" + +on: + push: + branches: [ '**'] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '27 4 * * 2' + +jobs: + + fedora: + name: fedora Build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: ['32', '33'] + container: + image: gpsbabel-docker.jfrog.io/tsteven4/gpsbabel_build_environment_f${{ matrix.version }} + env: + LC_ALL: C.UTF-8 + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: build_and_test + run: | + ./build_and_test diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 000000000..496397364 --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,87 @@ +name: "macos" + +on: + push: + branches: [ '**'] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '27 4 * * 2' + +jobs: + macos_release: + name: macos release Build + runs-on: macos-latest + env: + QT_VERSION: '5.12.10' + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Cache Qt + uses: actions/cache@v2 + with: + path: ~/Cache + key: ${{ runner.os }}-${{ env.QT_VERSION }}-${{ secrets.CACHE_VERSION }} + + - name: Qt install + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + run: | + ./tools/travis_install_osx ${QT_VERSION} installer + + - name: Script + run: | + source ${HOME}/Cache/qt-${QT_VERSION}.env + sudo xcode-select --switch /Applications/Xcode_11.7.app + ./tools/travis_script_osx + echo GITHUB_REF: $GITHUB_REF + + - name: Deploy + # This only handles continous releases now, for other events artifacts may be saved in + # the 'Upload Artifacts' step. + if: ( github.event_name == 'push' ) && ( github.ref == 'refs/heads/master' ) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_NAME: Continuous-${{ runner.os }} + run: | + ./tools/uploadtool/upload_github.sh gui/GPSBabel-*.dmg + + - name: 'Upload Artifacts' + uses: actions/upload-artifact@v2 + with: + name: MacOS_Installer + path: gui/GPSBabel-*.dmg + retention-days: 14 + + macos_edge: + name: macos edge Build + runs-on: macos-latest + env: + QT_VERSION: '5.15.2' + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Cache Qt + uses: actions/cache@v2 + with: + path: ~/Cache + key: ${{ runner.os }}-${{ env.QT_VERSION }}-${{ secrets.CACHE_VERSION }} + + - name: Qt install + env: + ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }} + ARTIFACTORY_API_KEY: ${{ secrets.ARTIFACTORY_API_KEY }} + ARTIFACTORY_BASE_URL: ${{ secrets.ARTIFACTORY_BASE_URL }} + run: | + ./tools/travis_install_osx ${QT_VERSION} artifactory + + - name: Script + run: | + source ${HOME}/Cache/qt-${QT_VERSION}.env + sudo xcode-select --switch /Applications/Xcode_12.1.1.app + ./tools/travis_script_osx diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml new file mode 100644 index 000000000..08440e47a --- /dev/null +++ b/.github/workflows/ubuntu.yml @@ -0,0 +1,96 @@ +name: "ubuntu" + +on: + push: + branches: [ '**'] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '27 4 * * 2' + +jobs: + basic: + name: basic Build + runs-on: ubuntu-latest + container: + image: gpsbabel-docker.jfrog.io/tsteven4/gpsbabel_build_environment + env: + LC_ALL: 'C.UTF-8' + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: build_and_test + run: | + ./build_and_test + + qtio_gcc: + name: qtio gcc Build + runs-on: ubuntu-latest + container: + image: gpsbabel-docker.jfrog.io/tsteven4/gpsbabel_build_environment_qtio + env: + LC_ALL: 'C.UTF-8' + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: build_and_test + run: | + . /opt/qtio.env + ./build_and_test + + qtio_clang: + name: qtio clang Build + runs-on: ubuntu-latest + container: + image: gpsbabel-docker.jfrog.io/tsteven4/gpsbabel_build_environment_qtio + env: + QMAKESPEC: 'linux-clang' + LC_ALL: 'C.UTF-8' + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: build_and_test + run: | + . /opt/qtio.env + ./build_and_test + + advanced: + name: advanced Build + runs-on: ubuntu-latest + container: + image: gpsbabel-docker.jfrog.io/tsteven4/gpsbabel_build_environment_focal + env: + LC_ALL: 'C.UTF-8' + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: build_extra_tests + run: | + ./tools/build_extra_tests + + coverage: + name: coverage Build + runs-on: ubuntu-20.04 + steps: + - name: install + run: | + sudo apt-get update + sudo apt-get install gcovr lcov libusb-1.0-0-dev qt5-default + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: script + env: + CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} + run: | + ./tools/travis_script_linux_coverage diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml new file mode 100644 index 000000000..446e62a51 --- /dev/null +++ b/.github/workflows/windows.yml @@ -0,0 +1,121 @@ +name: "windows" + +on: + push: + branches: [ '**'] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '27 4 * * 2' + +jobs: + + windows: + name: windows Build + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - QT_VERSION: '5.12.10' + ARCH: 'amd64' + HOST_ARCH: 'x86' + COMPILER: 'msvc2017_64' + FLOW: 'nmake' + RELEASE: true + - QT_VERSION: '5.12.10' + ARCH: 'amd64' + HOST_ARCH: 'x86' + COMPILER: 'msvc2017_64' + FLOW: 'msbuild' + RELEASE: false + - QT_VERSION: '5.12.10' + ARCH: 'amd64' + HOST_ARCH: 'x86' + COMPILER: 'msvc2017_64' + FLOW: 'cmake' + RELEASE: false + - QT_VERSION: '5.12.10' + ARCH: 'amd64' + HOST_ARCH: 'x86' + COMPILER: 'msvc2017_64' + VCVERSION: '14.16' + FLOW: 'nmake' + RELEASE: false + - QT_VERSION: '5.12.10' + ARCH: 'amd64' + HOST_ARCH: 'x86' + COMPILER: 'msvc2017_64' + VCVERSION: '14.0' + FLOW: 'nmake' + RELEASE: false + - QT_VERSION: '5.12.10' + ARCH: 'x86' + HOST_ARCH: 'x86' + COMPILER: 'msvc2017' + RELEASE: false + FLOW: 'nmake' + - QT_VERSION: '5.12.10' + ARCH: 'x86' + HOST_ARCH: 'x86' + COMPILER: 'msvc2017' + RELEASE: false + FLOW: 'msbuild' + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Cache Qt + uses: actions/cache@v2 + with: + path: ~/Cache + key: ${{ runner.os }}-${{ matrix.QT_VERSION }}-${{ matrix.COMPILER }}-${{ secrets.CACHE_VERSION }} + + - name: Install Qt + env: + CI_BUILD_DIR: ${{ github.workspace }} + shell: bash + run: | + ./tools/ci_install_windows.sh ${{ matrix.QT_VERSION }} ${{ matrix.COMPILER }} + + - name: Build + shell: powershell + run: | + .\tools\ci_setup_windows.ps1 -qtdir "$Home\Cache\Qt\${{ matrix.QT_VERSION }}\${{ matrix.COMPILER }}" -arch ${{ matrix.ARCH }} -host_arch ${{ matrix.HOST_ARCH }} -vcversion "${{ matrix.VCVERSION }}" + .\tools\make_windows_release.ps1 -iscc "C:\Program Files (x86)\Inno Setup 6\ISCC.exe" -buildinstaller true -flow ${{ matrix.FLOW }} -arch ${{ matrix.ARCH }} + + - name: Rename + shell: bash + run: | + VERSION_ID=$(date -u +%Y%m%dT%H%MZ --date=$(git show -s --format=%aI HEAD))-$(git rev-parse --short=7 HEAD) + mv gui/release/GPSBabel-*-Setup.exe gui/release/GPSBabel-${VERSION_ID}-Setup.exe + mv gui/release/GPSBabel-*-Manifest.txt gui/release/GPSBabel-${VERSION_ID}-Manifest.txt + + - name: Test + shell: bash + run: | + PATH="${HOME}/Cache/Qt/${{ matrix.QT_VERSION }}/${{ matrix.COMPILER }}/bin:${PATH}" + PNAME=./release/GPSBabel.exe GBTEMP=./gbtemp ./testo 2>&1 + PNAME=./release/GPSBabel.exe GBTEMP=./gbtemp ./test_encoding_utf8 2>&1 + + - name: Deploy + # This only handles continous releases now, for other events artifacts may be saved in + # the 'Upload Artifacts' step. + if: ( github.event_name == 'push' ) && ( github.ref == 'refs/heads/master' ) && matrix.RELEASE + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_NAME: Continuous-${{ runner.os }} + run: | + ./tools/uploadtool/upload_github.sh gui/release/GPSBabel-*-Setup.exe + + - name: 'Upload Artifacts' + uses: actions/upload-artifact@v2 + with: + name: Windows_Installer ${{ join(matrix.*) }} + path: | + ./gui/release/GPSBabel-*-Setup.exe + ./gui/release/GPSBabel-*-Manifest.txt + retention-days: 7 diff --git a/appveyor.yml b/appveyor.yml index 1b0e4bbc7..0345de381 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -40,7 +40,7 @@ build_script: if (($env:flow -eq "nmake") -and ($env:arch -eq "x86_amd64") -and ($env:qt -eq "5.12\msvc2017_64")) { # full build and deploy of our release configuration - & ".\tools\make_windows_release.ps1" -flow $env:flow -buildinstaller "true" + & ".\tools\make_windows_release.ps1" -flow $env:flow -buildinstaller "true" -arch $env:arch $verid=(Get-Date -UFormat "%Y%m%dT%H%MZ" -Date (Get-Date -Date (git show -s --format="%aI" HEAD)).toUniversalTime()) ` + '-' + (git rev-parse --short=7 HEAD) Get-ChildItem .\gui\release\GPSBabel-*-Setup.exe | % { Push-AppveyorArtifact $_.FullName -FileName "GPSBabel-$($verid)-Setup.exe" } @@ -48,7 +48,7 @@ build_script: } else { - & ".\tools\make_windows_release.ps1" -flow $env:flow + & ".\tools\make_windows_release.ps1" -flow $env:flow -arch $env:arch } # work around issue with standard error output causing a build failure by making sure last command succeeds. # see https://help.appveyor.com/discussions/problems/10014-false-build-fail-status @@ -98,4 +98,3 @@ deploy_script: # qt: 5.9\msvc2015 skip_tags: true - diff --git a/gui/setup.iss b/gui/setup.iss index f754da4bc..e82a5fb67 100644 --- a/gui/setup.iss +++ b/gui/setup.iss @@ -7,11 +7,8 @@ ; Uses the Inno setup compiler. ; windeployqt should be run to prepare the necessary Qt files before ; running Inno Setup. -#ifndef gui_build_dir_name - #define gui_build_dir_name "build-app-Desktop_Qt_5_5_1_MinGW_32bit-Release" -#endif -#ifndef gpsbabel_build_dir_name - #define gpsbabel_build_dir_name "build-GPSBabel-Desktop_Qt_5_5_1_MinGW_32bit-Release" +#ifndef package_dir + #define package_dir "..\build-app-Desktop-Release\package" #endif [Setup] @@ -47,10 +44,10 @@ Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{ Source: gmapbase.html; DestDir: "{app}"; Flags: ignoreversion Source: qt.conf; DestDir: "{app}"; Flags: ignoreversion -Source: "..\{#gui_build_dir_name}\release\*"; Excludes: "app.res,vc_redist.*.exe,*.cpp,*.h,*.o,*.obj"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs -Source: "..\{#gui_build_dir_name}\release\vc_redist.x86.exe"; DestDir: "{app}"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall -Source: "..\{#gui_build_dir_name}\release\vc_redist.x64.exe"; DestDir: "{app}"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall -Source: "..\..\{#gpsbabel_build_dir_name}\release\gpsbabel.exe"; DestDir: "{app}"; Flags: ignoreversion +Source: "{#package_dir}\*"; Excludes: "vc_redist.*.exe"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs +Source: "{#package_dir}\vc_redist.x86.exe"; DestDir: "{app}"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall +Source: "{#package_dir}\vc_redist.x64.exe"; DestDir: "{app}"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall +Source: "{#package_dir}\gpsbabel.exe"; DestDir: "{app}"; Flags: ignoreversion ; Source: release\help\*; DestDir: "{app}\help"; Flags: ignoreversion recursesubdirs createallsubdirs ; Translation strings extracted from source code. Include it in the dist diff --git a/gui/setup.iss.in b/gui/setup.iss.in index 6a4fdc65b..09560192d 100644 --- a/gui/setup.iss.in +++ b/gui/setup.iss.in @@ -7,11 +7,8 @@ ; Uses the Inno setup compiler. ; windeployqt should be run to prepare the necessary Qt files before ; running Inno Setup. -#ifndef gui_build_dir_name - #define gui_build_dir_name "build-app-Desktop_Qt_5_5_1_MinGW_32bit-Release" -#endif -#ifndef gpsbabel_build_dir_name - #define gpsbabel_build_dir_name "build-GPSBabel-Desktop_Qt_5_5_1_MinGW_32bit-Release" +#ifndef package_dir + #define package_dir "..\build-app-Desktop-Release\package" #endif [Setup] @@ -47,10 +44,10 @@ Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{ Source: gmapbase.html; DestDir: "{app}"; Flags: ignoreversion Source: qt.conf; DestDir: "{app}"; Flags: ignoreversion -Source: "..\{#gui_build_dir_name}\release\*"; Excludes: "app.res,vc_redist.*.exe,*.cpp,*.h,*.o,*.obj"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs -Source: "..\{#gui_build_dir_name}\release\vc_redist.x86.exe"; DestDir: "{app}"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall -Source: "..\{#gui_build_dir_name}\release\vc_redist.x64.exe"; DestDir: "{app}"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall -Source: "..\..\{#gpsbabel_build_dir_name}\release\gpsbabel.exe"; DestDir: "{app}"; Flags: ignoreversion +Source: "{#package_dir}\*"; Excludes: "vc_redist.*.exe"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs +Source: "{#package_dir}\vc_redist.x86.exe"; DestDir: "{app}"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall +Source: "{#package_dir}\vc_redist.x64.exe"; DestDir: "{app}"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall +Source: "{#package_dir}\gpsbabel.exe"; DestDir: "{app}"; Flags: ignoreversion ; Source: release\help\*; DestDir: "{app}\help"; Flags: ignoreversion recursesubdirs createallsubdirs ; Translation strings extracted from source code. Include it in the dist diff --git a/gui/setup.iss.qmake.in b/gui/setup.iss.qmake.in index 27672b86c..4ffb4c72a 100644 --- a/gui/setup.iss.qmake.in +++ b/gui/setup.iss.qmake.in @@ -7,11 +7,8 @@ ; Uses the Inno setup compiler. ; windeployqt should be run to prepare the necessary Qt files before ; running Inno Setup. -#ifndef gui_build_dir_name - #define gui_build_dir_name \"build-app-Desktop_Qt_5_5_1_MinGW_32bit-Release\" -#endif -#ifndef gpsbabel_build_dir_name - #define gpsbabel_build_dir_name \"build-GPSBabel-Desktop_Qt_5_5_1_MinGW_32bit-Release\" +#ifndef package_dir + #define package_dir \"..\\build-app-Desktop-Release\\package\" #endif [Setup] @@ -47,10 +44,10 @@ Name: \"desktopicon\"; Description: \"{cm:CreateDesktopIcon}\"; GroupDescription Source: gmapbase.html; DestDir: \"{app}\"; Flags: ignoreversion Source: qt.conf; DestDir: \"{app}\"; Flags: ignoreversion -Source: \"..\\{#gui_build_dir_name}\\release\\*\"; Excludes: \"app.res,vc_redist.*.exe,*.cpp,*.h,*.o,*.obj\"; DestDir: \"{app}\"; Flags: ignoreversion recursesubdirs createallsubdirs -Source: \"..\\{#gui_build_dir_name}\\release\\vc_redist.x86.exe\"; DestDir: \"{app}\"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall -Source: \"..\\{#gui_build_dir_name}\\release\\vc_redist.x64.exe\"; DestDir: \"{app}\"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall -Source: \"..\\..\\{#gpsbabel_build_dir_name}\\release\\gpsbabel.exe\"; DestDir: \"{app}\"; Flags: ignoreversion +Source: \"{#package_dir}\\*\"; Excludes: \"vc_redist.*.exe\"; DestDir: \"{app}\"; Flags: ignoreversion recursesubdirs createallsubdirs +Source: \"{#package_dir}\\vc_redist.x86.exe\"; DestDir: \"{app}\"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall +Source: \"{#package_dir}\\vc_redist.x64.exe\"; DestDir: \"{app}\"; Flags: ignoreversion skipifsourcedoesntexist deleteafterinstall +Source: \"{#package_dir}\\gpsbabel.exe\"; DestDir: \"{app}\"; Flags: ignoreversion ; Source: release\\help\\*; DestDir: \"{app}\\help\"; Flags: ignoreversion recursesubdirs createallsubdirs ; Translation strings extracted from source code. Include it in the dist diff --git a/tools/Dockerfile b/tools/Dockerfile index a0323885b..7198937df 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -7,10 +7,13 @@ LABEL maintainer="https://github.com/tsteven4" WORKDIR /app # update environment. +# bionic needs a newer git than provided by ubuntu:bionic ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ apt-utils \ + software-properties-common \ && apt-get upgrade -y \ + && add-apt-repository ppa:git-core/ppa \ && rm -rf /var/lib/apt/lists/* # install packages needed for gpsbabel build @@ -68,4 +71,3 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && sed -i 's/^# *\(en_US ISO-8859-1\)/\1/' /etc/locale.gen \ && locale-gen \ && locale -a - diff --git a/tools/Dockerfile_f32 b/tools/Dockerfile_f32 index ee55b7c3f..03bde6c7d 100644 --- a/tools/Dockerfile_f32 +++ b/tools/Dockerfile_f32 @@ -20,4 +20,3 @@ RUN dnf install --assumeyes expat desktop-file-utils libxslt docbook-style-xsl f dnf clean all # create a link as fedora uses the name qmake-qt5 for Qt5's qmake. RUN alternatives --install /usr/bin/qmake qt /usr/lib64/qt5/bin/qmake 100 - diff --git a/tools/Dockerfile_f33 b/tools/Dockerfile_f33 new file mode 100644 index 000000000..41a6b95d2 --- /dev/null +++ b/tools/Dockerfile_f33 @@ -0,0 +1,22 @@ +# this file is used to build the image gpsbabel_build_environment used by travis. + +FROM fedora:33 + +LABEL maintainer="https://github.com/tsteven4" + +WORKDIR /app + +# basic tools to build +RUN dnf install --assumeyes git make valgrind diffutils which findutils langpacks-en && \ + dnf clean all +# libraries used by gpsbabel. zlib and shapelib may or may not be used depending qmake options. +RUN dnf install --assumeyes libusb-devel zlib-devel shapelib-devel && \ + dnf clean all +# Qt used by gpsbabel, gpsbabelfe +RUN dnf install --assumeyes qt5-qtbase-devel qt5-qtwebengine-devel qt5-linguist qt5-qttranslations && \ + dnf clean all +# tools to build the docs +RUN dnf install --assumeyes expat desktop-file-utils libxslt docbook-style-xsl fop && \ + dnf clean all +# create a link as fedora uses the name qmake-qt5 for Qt5's qmake. +RUN alternatives --install /usr/bin/qmake qt /usr/lib64/qt5/bin/qmake 100 diff --git a/tools/Dockerfile_qtio b/tools/Dockerfile_qtio index 8735dd32d..15b8e9712 100644 --- a/tools/Dockerfile_qtio +++ b/tools/Dockerfile_qtio @@ -41,10 +41,13 @@ WORKDIR /app ARG QT_VERSION # update environment. +# bionic needs a newer git than provided by ubuntu:bionic ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ apt-utils \ + software-properties-common \ && apt-get upgrade -y \ + && add-apt-repository ppa:git-core/ppa \ && rm -rf /var/lib/apt/lists/* # install packages needed for gpsbabel build @@ -109,4 +112,3 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && sed -i 's/^# *\(en_US ISO-8859-1\)/\1/' /etc/locale.gen \ && locale-gen \ && locale -a - diff --git a/tools/ci_install_windows.sh b/tools/ci_install_windows.sh new file mode 100755 index 000000000..0a3c1655a --- /dev/null +++ b/tools/ci_install_windows.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -ex + +# validate install +function validate() { + ( + set +e + # shellcheck source=/dev/null + source "${CACHEDIR}/qt.env" + if [ "$(cygpath -u "$(qmake -query QT_INSTALL_BINS)")" != "${QTDIR}/bin" ]; then + echo "ERROR: unexpected Qt location." >&2 + exit 1 + fi + if [ "$(qmake -query QT_VERSION)" != "${QT_VERSION}" ]; then + echo "ERROR: wrong Qt version." >&2 + exit 1 + fi + ) +} + +QT_VERSION=${1:-5.12.10} +COMPILER=${2:-msvc2017_64} +if [ "${COMPILER}" = "msvc2017_64" ]; then + PACKAGE_SUFFIX=win64_msvc2017_64 +elif [ "${COMPILER}" = "msvc2017" ]; then + PACKAGE_SUFFIX=win32_msvc2017 +else + echo "ERROR: unrecognized Qt compiler ${COMPILER}." >&2 + exit 1 +fi + +CACHEDIR=${HOME}/Cache +QTDIR=${CACHEDIR}/Qt/${QT_VERSION}/${COMPILER} + +if [ -d "${QTDIR}/bin" ]; then + echo "Using cached Qt." +else + rm -fr "${CACHEDIR}" + QT_VERSION_SHORT=${QT_VERSION//./} + curl -s -L -o "qt-opensource-windows-x86-${QT_VERSION}.exe" "https://download.qt.io/official_releases/qt/5.12/${QT_VERSION}/qt-opensource-windows-x86-${QT_VERSION}.exe" + ls -l ./*.exe + netsh advfirewall firewall add rule name=dummyupqt dir=out action=block program="$(cygpath -w "${PWD}/qt-opensource-windows-x86-${QT_VERSION}.exe")" + "${PWD}/qt-opensource-windows-x86-${QT_VERSION}.exe" --verbose --script "${CI_BUILD_DIR}/tools/qtci/qt-install.qs" QTCI_OUTPUT="${CACHEDIR}/Qt" QTCI_PACKAGES="qt.qt5.${QT_VERSION_SHORT}.${PACKAGE_SUFFIX},qt.qt5.${QT_VERSION_SHORT}.qtwebengine" + netsh advfirewall firewall delete rule name=dummyupqt + rm "qt-opensource-windows-x86-${QT_VERSION}.exe" + ls "${CACHEDIR}/Qt" + rm -fr "${CACHEDIR}/Qt/Docs" + rm -fr "${CACHEDIR}/Qt/Examples" + rm -fr "${CACHEDIR}/Qt/Tools" + rm -f "${CACHEDIR}/Qt/MaintenanceTool.*" + echo "export PATH=${QTDIR}/bin:\$PATH" > "${CACHEDIR}/qt.env" +fi +validate diff --git a/tools/ci_setup_windows.ps1 b/tools/ci_setup_windows.ps1 new file mode 100755 index 000000000..582a84c08 --- /dev/null +++ b/tools/ci_setup_windows.ps1 @@ -0,0 +1,55 @@ +# set up environmental variables for Qt and MSVC + +# Some tools only provide batch script files for setting up the environment. We +# run these batch scripts and then (re)set all environmental variables from the +# command shell enviornment in the power shell environment. +# see https://github.com/actions/virtual-environments/issues/294 +# and https://github.com/microsoft/vswhere/wiki/Start-Developer-Command-Prompt + +Param( + [string] $qtdir = "C:\Qt\Qt5.12.10\5.12.10\msvc2017_64", + [ValidateSet("x86", "amd64")][string] $arch = "amd64", + [ValidateSet("x86", "amd64")][string] $host_arch = "amd64", + [string] $vcversion +) + +# This is a bit of overkill. All we really need from qtenv2.bat is the path to +# the Qt bin directory, which we know in order to find qtenv2.bat! +# But running qtenv2.bat is what the Qt command shell shortcuts placed on +# the start menu by Qt do. +# TODO: check for an error when the bat file is run. +function Invoke-QtEnvironment($installationPath) { + $Command = Join-Path $installationPath "bin\qtenv2.bat" + & "${env:COMSPEC}" /s /c "`"$Command`" && set" | ForEach-Object { + if ($_ -match '^([^=]+)=(.*)') { + [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2]) + } + } +} + +# TODO: check for an error when the bat file is run. +# One way to generate an error is to request a vcvars_ver version that isn't +# available. +function Invoke-VSDevEnvironment($arch, $host_arch, $vcversion) { + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + $installationPath = & $vswhere -legacy -latest -property installationPath + if ( $vcversion ) { + $vcvars_ver = "-vcvars_ver=$vcversion" + } + $Command = Join-Path $installationPath "Common7\Tools\vsdevcmd.bat" + & "${env:COMSPEC}" /s /c "`"$Command`" -no_logo -arch=$arch -host_arch=$host_arch $vcvars_ver && set" | ForEach-Object { + if ($_ -match '^([^=]+)=(.*)') { + [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2]) + } + } +} + +$ErrorActionPreference = "Stop" + +Invoke-QtEnvironment $qtdir +# verify qmake can be found. +Get-Command qmake.exe | Format-Table -AutoSize -Wrap + +Invoke-VSDevEnvironment -arch $arch -host_arch $host_arch -vcversion $vcversion +# verify the c compiler can be found. +Get-Command cl.exe | Format-Table -AutoSize -Wrap diff --git a/tools/make_windows_release.ps1 b/tools/make_windows_release.ps1 index b91d2c60c..236e3ec1f 100755 --- a/tools/make_windows_release.ps1 +++ b/tools/make_windows_release.ps1 @@ -8,63 +8,87 @@ # powershell.exe -ExecutionPolicy Unrestricted -File tools\make_windows_release.ps1 -iscc "C:\Program Files (x86)\Inno Setup 5\ISCC.exe" -gpsbabel_build_dir_name "build-GPSBabel-Desktop_Qt_5_9_3_MSVC2015_64bit-Release" -gui_build_dir_name "build-app-Desktop_Qt_5_9_3_MSVC2015_64bit-Release" # # Be aware this script is used by appveyor.yml -# +# # The defaults should be compatible with appveyor builds. Param( - $windeployqt = "windeployqt.exe", - $iscc = "C:\Program Files (x86)\Inno Setup 5\ISCC.exe", - $gpsbabel_build_dir_name = "build-GPSBabel-Desktop-Release", - $gui_build_dir_name = "build-app-Desktop-Release", - $flow = "nmake", - $buildinstaller = "false" + $windeployqt = "windeployqt.exe", + $iscc = "C:\Program Files (x86)\Inno Setup 5\ISCC.exe", + $gpsbabel_build_dir_name = "build-GPSBabel-Desktop-Release", + $gui_build_dir_name = "build-app-Desktop-Release", + [ValidateSet("mingw", "msbuild", "nmake", "cmake")] $flow = "nmake", + $buildinstaller = "false", + [ValidateSet("x86", "amd64", "amd64_x86", "x86_amd64")] $arch = "amd64" ) +# the arch parameter values correspond to: +# vcvarsall arch parameter x86 => host x86, target x86. +# vcvarsall arch paramter amd64 => host amd64, target amd64. +# vcvarsall arch parameter amd64_x86 => host amd64, target x86 +# vcvarsall arch parameter x86_amd64 => host x86, target amd64 +# vsdevcmd arch parameter x86 => target x86. +# vsdevcmd arch parameter amd64 => target amd64. $ErrorActionPreference = "Stop" # verify we are in the top of the gpsbabel clone Get-Item tools/make_windows_release.ps1 -ErrorAction Stop | Out-Null $gpsbabel_src_dir = "$Pwd" $gpsbabel_build_dir = "$($gpsbabel_src_dir)\..\$($gpsbabel_build_dir_name)" $gui_build_dir = "$($gpsbabel_src_dir)\$($gui_build_dir_name)" +if ( "$flow" -eq "msbuild" ) { + # translate target architecture to Platform property value. + switch ($arch) { + "x86" { $platform = "Win32" } + "amd64" { $platform = "x64" } + "amd64_x86" { $platform = "Win32" } + "x86_amd64" { $platform = "x64" } + } +} +if ( "$flow" -eq "cmake" ) { + $Qt5_DIR = "$(Join-Path "$((Get-Command qmake) | Split-Path)" '..\lib\cmake\Qt5' -Resolve)" +} # mimic creator shadow build to match Inno setup file # make sure we are staring with a clean build directory Remove-Item "$($gpsbabel_build_dir)" -Recurse -ErrorAction Ignore -New-Item "$($gpsbabel_build_dir)" -type directory -force | Out-Null +New-Item "$($gpsbabel_build_dir)" -type directory -Force | Out-Null Set-Location "$($gpsbabel_build_dir)" switch ($flow) { - "mingw" { qmake "$($gpsbabel_src_dir)\GPSBabel.pro" -spec "win32-g++" } - # work around Qt 5.12.1, 5.12.2 qmake bug. - #C:\Qt\5.12.1\msvc2017\bin\qmake.exe -tp vc GPSBabel.pro - #WARNING: Could not parse Compiler option '-std:c++14'; added to AdditionalOptions. - #WARNING: You can suppress these warnings with CONFIG+=suppress_vcproj_warnings. - #WARNING: Could not parse Compiler option '-std:c++14'; added to AdditionalOptions. - "msbuild" { $ErrorActionPreference = "Continue"; qmake -tp vc "$($gpsbabel_src_dir)\GPSBabel.pro"; $ErrorActionPreference = "Stop" } - "nmake" { qmake "$($gpsbabel_src_dir)\GPSBabel.pro" -spec "win32-msvc" } + "mingw" { qmake "$($gpsbabel_src_dir)\GPSBabel.pro" -spec "win32-g++" } + # work around Qt 5.12.1, 5.12.2 qmake bug. + #C:\Qt\5.12.1\msvc2017\bin\qmake.exe -tp vc GPSBabel.pro + #WARNING: Could not parse Compiler option '-std:c++14'; added to AdditionalOptions. + #WARNING: You can suppress these warnings with CONFIG+=suppress_vcproj_warnings. + #WARNING: Could not parse Compiler option '-std:c++14'; added to AdditionalOptions. + "msbuild" { $ErrorActionPreference = "Continue"; qmake -tp vc "$($gpsbabel_src_dir)\GPSBabel.pro"; $ErrorActionPreference = "Stop" } + "nmake" { qmake "$($gpsbabel_src_dir)\GPSBabel.pro" -spec "win32-msvc" } + "cmake" { cmake -G "Ninja" -DCMAKE_BUILD_TYPE:STRING="Release" -DQt5_DIR:PATH="$($Qt5_DIR)" -DCMAKE_RUNTIME_OUTPUT_DIRECTORY:PATH="$($gpsbabel_build_dir)\release" "$($gpsbabel_src_dir)" } } if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) } switch ($flow) { - "mingw" { ming32-make } - "msbuild" { msbuild GPSBabel.vcxproj -property:Configuration=Release } - "nmake" { nmake /NOLOGO } + "mingw" { ming32-make } + "msbuild" { msbuild GPSBabel.vcxproj -property:Configuration=Release -property:Platform=$platform } + "nmake" { nmake /NOLOGO } + "cmake" { cmake --build . } } if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) } # copy GPSBabel.exe for use by test_script Remove-Item "$($gpsbabel_src_dir)\release" -Recurse -ErrorAction Ignore -New-Item "$($gpsbabel_src_dir)\release" -type directory -force | Out-Null +New-Item "$($gpsbabel_src_dir)\release" -type directory -Force | Out-Null Copy-Item release\GPSBabel.exe "$($gpsbabel_src_dir)\release\GPSBabel.exe" Set-Location "$($gpsbabel_src_dir)" # make sure we are staring with a clean build directory Remove-Item "$($gui_build_dir)" -Recurse -ErrorAction Ignore -New-Item "$($gui_build_dir)" -type directory -force | Out-Null +New-Item "$($gui_build_dir)" -type directory -Force | Out-Null Set-Location "$($gui_build_dir)" switch ($flow) { - "mingw" { qmake "$($gpsbabel_src_dir)\gui\app.pro" -spec "win32-g++" } - "msbuild" { qmake -tp vc "$($gpsbabel_src_dir)\gui\app.pro"} - "nmake" { qmake "$($gpsbabel_src_dir)\gui\app.pro" -spec "win32-msvc" } + "mingw" { qmake "$($gpsbabel_src_dir)\gui\app.pro" -spec "win32-g++" } + "msbuild" { qmake -tp vc "$($gpsbabel_src_dir)\gui\app.pro" } + "nmake" { qmake "$($gpsbabel_src_dir)\gui\app.pro" -spec "win32-msvc" } + "cmake" { cmake -G "Ninja" -DCMAKE_BUILD_TYPE:STRING:="Release" -DQt5_DIR:PATH="$($Qt5_DIR)" -DCMAKE_RUNTIME_OUTPUT_DIRECTORY:PATH="$($gui_build_dir)\release" "$($gpsbabel_src_dir)\gui" } } if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) } switch ($flow) { - "mingw" { ming32-make } - "msbuild" { msbuild GPSBabelFE.vcxproj -property:Configuration=Release } - "nmake" { nmake /NOLOGO } + "mingw" { ming32-make } + "msbuild" { msbuild GPSBabelFE.vcxproj -property:Configuration=Release -property:Platform=$platform } + "nmake" { nmake /NOLOGO } + "cmake" { cmake --build . } } if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) } # work around errors with lupdate, lrelease misprocessing qtHaveModule(webenginewidgets) @@ -75,13 +99,17 @@ $ErrorActionPreference = "Continue" lupdate "$($gpsbabel_src_dir)\gui\app.pro" lrelease "$($gpsbabel_src_dir)\gui\app.pro" $ErrorActionPreference = "Stop" +# deploy to a clean directory as different build systems create differently named debris in release. +Remove-Item "$($gui_build_dir)\package" -Recurse -ErrorAction Ignore +New-Item "$($gui_build_dir)\package" -type directory -Force | Out-Null +Copy-Item "$($gpsbabel_build_dir)\release\GPSBabel.exe" "$($gui_build_dir)\package\GPSBabel.exe" +Copy-Item "$($gui_build_dir)\release\GPSBabelFE.exe" "$($gui_build_dir)\package\GPSBabelFE.exe" # use --plugindir option to locate the plugins. -& "$($windeployqt)" --verbose 10 --plugindir release\plugins release\GPSBabelFE.exe +& "$($windeployqt)" --verbose 1 --plugindir package\plugins package\GPSBabelFE.exe if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) } -if ($buildinstaller -eq "true") -{ - Set-Location "$($gpsbabel_src_dir)\gui" - & "$($iscc)" /Dgpsbabel_build_dir_name="$($gpsbabel_build_dir_name)" /Dgui_build_dir_name="$($gui_build_dir_name)" setup.iss - if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) } +if ($buildinstaller -eq "true") { + Set-Location "$($gpsbabel_src_dir)\gui" + & "$($iscc)" /Dpackage_dir="$($gui_build_dir)\package" setup.iss + if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) } } Set-Location "$($gpsbabel_src_dir)" diff --git a/tools/uploadtool/upload_github.sh b/tools/uploadtool/upload_github.sh new file mode 100755 index 000000000..c67a58923 --- /dev/null +++ b/tools/uploadtool/upload_github.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +set +x # Do not leak information + +# Exit immediately if one of the files given as arguments is not there +# because we don't want to delete the existing release if we don't have +# the new files that should be uploaded +for file in "$@" +do + if [ ! -e "$file" ] + then echo "$file is missing, giving up." >&2; exit 1 + fi +done + +if [ $# -eq 0 ]; then + echo "No artifacts to use for release, giving up." + exit 0 +fi + +if command -v sha256sum >/dev/null 2>&1 ; then + shatool="sha256sum" +elif command -v shasum >/dev/null 2>&1 ; then + shatool="shasum -a 256" # macOS fallback +else + echo "Neither sha256sum nor shasum is available, cannot check hashes" +fi + +if [ "${GITHUB_ACTIONS}" = "true" ]; then + # We are running on GitHub Actions + echo "Running on GitHub Actions" + echo "GITHUB_SHA: ${GITHUB_SHA}" +else + echo "This script only works with GitHub Actions." + exit 1 +fi + +# Do not use "latest" as it is reserved by GitHub +RELEASE_NAME=${RELEASE_NAME:-continuous} +RELEASE_TITLE="Continuous build" +is_prerelease="true" + +if [ "${GITHUB_EVENT_NAME}" == "pull_request" ] ; then + echo "Release uploading disabled for pull requests, uploading to transfersh.com instead" + rm -f ./uploaded-to + for FILE in "$@" ; do + BASENAME="$(basename "${FILE}")" + curl --upload-file $FILE "https://transfersh.com/$BASENAME" > ./one-upload + echo "$(cat ./one-upload)" # this way we get a newline + echo -n "$(cat ./one-upload)\\n" >> ./uploaded-to # this way we get a \n but no newline + done + $shatool "$@" + exit 0 +fi + +REPO_SLUG="${GITHUB_REPOSITORY}" +if [ -z "${GITHUB_TOKEN}" ] ; then + echo "GITHUB_TOKEN missing, please set it in the GitHub Settings/Secrets of this repository." + echo "You can get one from https://github.com/settings/tokens" + exit 1 +fi + +tag_url="https://api.github.com/repos/$REPO_SLUG/git/refs/tags/$RELEASE_NAME" +tag_infos=$(curl -XGET --header "Authorization: token ${GITHUB_TOKEN}" "${tag_url}") +echo "tag_infos: $tag_infos" +tag_sha=$(echo "$tag_infos" | grep '"sha":' | head -n 1 | cut -d '"' -f 4 | cut -d '{' -f 1) +echo "tag_sha: $tag_sha" + +release_url="https://api.github.com/repos/$REPO_SLUG/releases/tags/$RELEASE_NAME" +echo "Getting the release ID..." +echo "release_url: $release_url" +release_infos=$(curl -XGET --header "Authorization: token ${GITHUB_TOKEN}" "${release_url}") +echo "release_infos: $release_infos" +release_id=$(echo "$release_infos" | grep "\"id\":" | head -n 1 | tr -s " " | cut -f 3 -d" " | cut -f 1 -d ",") +echo "release ID: $release_id" +upload_url=$(echo "$release_infos" | grep '"upload_url":' | head -n 1 | cut -d '"' -f 4 | cut -d '{' -f 1) +echo "upload_url: $upload_url" +release_url=$(echo "$release_infos" | grep '"url":' | head -n 1 | cut -d '"' -f 4 | cut -d '{' -f 1) +echo "release_url: $release_url" +target_commit_sha=$(echo "$release_infos" | grep '"target_commitish":' | head -n 1 | cut -d '"' -f 4 | cut -d '{' -f 1) +echo "target_commit_sha: $target_commit_sha" + +if [ "{$GITHUB_SHA}" != "$target_commit_sha" ] ; then + + echo "GITHUB_SHA != target_commit_sha, hence deleting $RELEASE_NAME..." + + if [ ! -z "$release_id" ]; then + delete_url="https://api.github.com/repos/$REPO_SLUG/releases/$release_id" + echo "Delete the release..." + echo "delete_url: $delete_url" + curl -XDELETE \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + "${delete_url}" + fi + + if [ "$RELEASE_NAME" == "continuous" ] ; then + # if this is a continuous build tag, then delete the old tag + # in preparation for the new release + echo "Delete the tag..." + delete_url="https://api.github.com/repos/$REPO_SLUG/git/refs/tags/$RELEASE_NAME" + echo "delete_url: $delete_url" + curl -XDELETE \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + "${delete_url}" + fi + + echo "Create release..." + + BODY="CI build log: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + + release_infos=$(curl -H "Authorization: token ${GITHUB_TOKEN}" \ + --data '{"tag_name": "'"$RELEASE_NAME"'","target_commitish": "'"${GITHUB_SHA}"'","name": "'"$RELEASE_TITLE"'","body": "'"$BODY"'","draft": false,"prerelease": '$is_prerelease'}' "https://api.github.com/repos/$REPO_SLUG/releases") + + echo "$release_infos" + + unset upload_url + upload_url=$(echo "$release_infos" | grep '"upload_url":' | head -n 1 | cut -d '"' -f 4 | cut -d '{' -f 1) + echo "upload_url: $upload_url" + + unset release_url + release_url=$(echo "$release_infos" | grep '"url":' | head -n 1 | cut -d '"' -f 4 | cut -d '{' -f 1) + echo "release_url: $release_url" + +fi # if [ "${GITHUB_SHA}" != "$tag_sha" ] + +if [ -z "$release_url" ] ; then + echo "Cannot figure out the release URL for $RELEASE_NAME" + exit 1 +fi + +echo "Upload binaries to the release..." + +for FILE in "$@" ; do + FULLNAME="${FILE}" + BASENAME="$(basename "${FILE}")" + curl -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Accept: application/vnd.github.manifold-preview" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @$FULLNAME \ + "$upload_url?name=$BASENAME" + echo "" +done + +$shatool "$@" + +if [ "${GITHUB_SHA}" != "$tag_sha" ] ; then + echo "Publish the release..." + + release_infos=$(curl -H "Authorization: token ${GITHUB_TOKEN}" \ + --data '{"draft": false}' "$release_url") + + echo "$release_infos" +fi # if [ "${GITHUB_SHA}" != "$tag_sha" ] -- 2.30.2